from pymongo import MongoClient
from bs4 import BeautifulSoup
from pprint import pprint
import requests
import datetime
import json
import time
import re

server_client = MongoClient('localhost', 27017)

server_db = server_client['knx_posts_db']
offical_posts_coll = server_db['offical_posts_coll']


class POST():
    def __init__(self):
        self.params = {

        }
        self.data = {

        }
        self.headers = {

        }
        self.scrapy()

    def scrapy(self):
        for i in range(1, 4):
            r = requests.get('http://www.tul.com.cn/cn/rencaizhongxin/zhaopinxinxi/p' + str(i))
            soup = BeautifulSoup(r.text)

            for jd in soup.select('.zpxx-list table tbody tr'):
                name = jd.select('td')[0].text
                date = jd.select('td')[1].text
                count = jd.select('td')[2].text
                edu = jd.select('td')[3].text

                url = 'http://www.tul.com.cn' + jd.find('a').get('href')
                r = requests.get(url)
                detail_soup = BeautifulSoup(r.text)

                major = detail_soup.find(text = re.compile('需求专业：')).parent.parent.get_text().split('：')[1]
                description0 = detail_soup.find(text = re.compile('岗位职责/岗位要求：')).parent.parent.get_text()
                description1 = detail_soup.find(text = re.compile('待遇及联系：')).parent.parent.get_text()

                if re.search('工作地点：([^<]+)', description1):
                    location = re.search('工作地点：([^</]+)', description1).group().split('\r')[0].split('：')[1].strip()
                elif re.search('公司地址：([^<]+)', description1):
                    location = re.search('公司地址：([^</]+)', description1).group().split('\r')[0].split('：')[1].strip()
                elif re.search('地址：([^<]+)', description1):
                    location = re.search('地址：([^</]+)', description1).group().split('\r')[0].split('：')[1].strip()

                item = {
                    "url": url,
                    'edu': edu,
                    'exp': [],
                    'name': name,
                    'date': date,
                    'lang': '',
                    'place': location,
                    'major': major,
                    'count': count,
                    'salary': [],
                    'toSchool': True,
                    'welfare': [],
                    'funType': '',
                    'company': '珠海联邦制药股份有限公司',
                    'location': location,
                    'industry': '医药',
                    'keywords': [],
                    'platform': 'offical',
                    'searchKeyword': '',
                    'description': (description0 + '\n' + description1).strip(),
                    'subIndustry': '',
                    'stime': datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
                }

                result = offical_posts_coll.replace_one({'company': '珠海联邦制药股份有限公司', 'name': name, 'location': location}, item, True)

                if result.matched_count:
                    print('-' * 40, 'update one job', '-' * 40)
                else:
                    print('-' * 40, 'insert one job', '-' * 40)

                pprint(item)


POST()
